<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - optimization_ex.cpp</title></head><body bgcolor='white'><pre>
<font color='#009900'>// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
</font><font color='#009900'>/*

    This is an example illustrating the use the general purpose non-linear
    optimization routines from the dlib C++ Library.

    The library provides implementations of many popular algorithms such as L-BFGS
    and BOBYQA.  These algorithms allow you to find the minimum or maximum of a
    function of many input variables.  This example walks though a few of the ways
    you might put these routines to use.

*/</font>


<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>optimization.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>dlib<font color='#5555FF'>/</font>global_optimization.h<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>iostream<font color='#5555FF'>&gt;</font>


<font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> std;
<font color='#0000FF'>using</font> <font color='#0000FF'>namespace</font> dlib;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#009900'>// In dlib, most of the general purpose solvers optimize functions that take a
</font><font color='#009900'>// column vector as input and return a double.  So here we make a typedef for a
</font><font color='#009900'>// variable length column vector of doubles.  This is the type we will use to
</font><font color='#009900'>// represent the input to our objective functions which we will be minimizing.
</font><font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font>,<font color='#979000'>0</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font> column_vector;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// Below we create a few functions.  When you get down into main() you will see that
</font><font color='#009900'>// we can use the optimization algorithms to find the minimums of these functions.
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>double</u></font> <b><a name='rosen'></a>rosen</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> m<font face='Lucida Console'>)</font>
<font color='#009900'>/*
    This function computes what is known as Rosenbrock's function.  It is 
    a function of two input variables and has a global minimum at (1,1).
    So when we use this function to test out the optimization algorithms
    we will see that the minimum found is indeed at the point (1,1). 
*/</font>
<b>{</b>
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> x <font color='#5555FF'>=</font> <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>; 
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> y <font color='#5555FF'>=</font> <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// compute Rosenbrock's function and return the result
</font>    <font color='#0000FF'>return</font> <font color='#979000'>100.0</font><font color='#5555FF'>*</font><font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font>y <font color='#5555FF'>-</font> x<font color='#5555FF'>*</font>x,<font color='#979000'>2</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>pow</font><font face='Lucida Console'>(</font><font color='#979000'>1</font> <font color='#5555FF'>-</font> x,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;
<b>}</b>

<font color='#009900'>// This is a helper function used while optimizing the rosen() function.  
</font><font color='#0000FF'>const</font> column_vector <b><a name='rosen_derivative'></a>rosen_derivative</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> m<font face='Lucida Console'>)</font>
<font color='#009900'>/*!
    ensures
        - returns the gradient vector for the rosen function
!*/</font>
<b>{</b>
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> x <font color='#5555FF'>=</font> <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> y <font color='#5555FF'>=</font> <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// make us a column vector of length 2
</font>    column_vector <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>2</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// now compute the gradient vector
</font>    <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>400</font><font color='#5555FF'>*</font>x<font color='#5555FF'>*</font><font face='Lucida Console'>(</font>y<font color='#5555FF'>-</font>x<font color='#5555FF'>*</font>x<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>x<font face='Lucida Console'>)</font>; <font color='#009900'>// derivative of rosen() with respect to x
</font>    <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>200</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>y<font color='#5555FF'>-</font>x<font color='#5555FF'>*</font>x<font face='Lucida Console'>)</font>;              <font color='#009900'>// derivative of rosen() with respect to y
</font>    <font color='#0000FF'>return</font> res;
<b>}</b>

<font color='#009900'>// This function computes the Hessian matrix for the rosen() fuction.  This is
</font><font color='#009900'>// the matrix of second derivatives.
</font>matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <b><a name='rosen_hessian'></a>rosen_hessian</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> m<font face='Lucida Console'>)</font>
<b>{</b>
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> x <font color='#5555FF'>=</font> <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>;
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> y <font color='#5555FF'>=</font> <font color='#BB00BB'>m</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

    matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>2</font>,<font color='#979000'>2</font><font face='Lucida Console'>)</font>;

    <font color='#009900'>// now compute the second derivatives 
</font>    <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1200</font><font color='#5555FF'>*</font>x<font color='#5555FF'>*</font>x <font color='#5555FF'>-</font> <font color='#979000'>400</font><font color='#5555FF'>*</font>y <font color='#5555FF'>+</font> <font color='#979000'>2</font>; <font color='#009900'>// second derivative with respect to x
</font>    <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>400</font><font color='#5555FF'>*</font>x;   <font color='#009900'>// derivative with respect to x and y
</font>    <font color='#BB00BB'>res</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,<font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>200</font>;                 <font color='#009900'>// second derivative with respect to y
</font>    <font color='#0000FF'>return</font> res;
<b>}</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'>class</font> <b><a name='rosen_model'></a>rosen_model</b> 
<b>{</b>
    <font color='#009900'>/*!
        This object is a "function model" which can be used with the
        find_min_trust_region() routine.  
    !*/</font>

<font color='#0000FF'>public</font>:
    <font color='#0000FF'>typedef</font> ::column_vector column_vector;
    <font color='#0000FF'>typedef</font> matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font> general_matrix;

    <font color='#0000FF'><u>double</u></font> <b><a name='operator'></a>operator</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x
    <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>rosen</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>; <b>}</b>

    <font color='#0000FF'><u>void</u></font> <b><a name='get_derivative_and_hessian'></a>get_derivative_and_hessian</b> <font face='Lucida Console'>(</font>
        <font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x,
        column_vector<font color='#5555FF'>&amp;</font> der,
        general_matrix<font color='#5555FF'>&amp;</font> hess
    <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
    <b>{</b>
        der <font color='#5555FF'>=</font> <font color='#BB00BB'>rosen_derivative</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
        hess <font color='#5555FF'>=</font> <font color='#BB00BB'>rosen_hessian</font><font face='Lucida Console'>(</font>x<font face='Lucida Console'>)</font>;
    <b>}</b>
<b>}</b>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<font color='#0000FF'><u>int</u></font> <b><a name='main'></a>main</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>try</font>
<b>{</b>
    <font color='#009900'>// Set the starting point to (4,8).  This is the point the optimization algorithm
</font>    <font color='#009900'>// will start out from and it will move it closer and closer to the function's 
</font>    <font color='#009900'>// minimum point.   So generally you want to try and compute a good guess that is
</font>    <font color='#009900'>// somewhat near the actual optimum value.
</font>    column_vector starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>4</font>, <font color='#979000'>8</font><b>}</b>;

    <font color='#009900'>// The first example below finds the minimum of the rosen() function and uses the
</font>    <font color='#009900'>// analytical derivative computed by rosen_derivative().  Since it is very easy to
</font>    <font color='#009900'>// make a mistake while coding a function like rosen_derivative() it is a good idea
</font>    <font color='#009900'>// to compare your derivative function against a numerical approximation and see if
</font>    <font color='#009900'>// the results are similar.  If they are very different then you probably made a 
</font>    <font color='#009900'>// mistake.  So the first thing we do is compare the results at a test point: 
</font>    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Difference between analytic derivative and numerical approximation of derivative: </font>" 
         <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>length</font><font face='Lucida Console'>(</font><font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font><font face='Lucida Console'>(</font>starting_point<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>rosen_derivative</font><font face='Lucida Console'>(</font>starting_point<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;


    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>Find the minimum of the rosen function()</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    <font color='#009900'>// Now we use the find_min() function to find the minimum point.  The first argument
</font>    <font color='#009900'>// to this routine is the search strategy we want to use.  The second argument is the 
</font>    <font color='#009900'>// stopping strategy.  Below I'm using the objective_delta_stop_strategy which just 
</font>    <font color='#009900'>// says that the search should stop when the change in the function being optimized 
</font>    <font color='#009900'>// is small enough.
</font>
    <font color='#009900'>// The other arguments to find_min() are the function to be minimized, its derivative, 
</font>    <font color='#009900'>// then the starting point, and the last is an acceptable minimum value of the rosen() 
</font>    <font color='#009900'>// function.  That is, if the algorithm finds any inputs to rosen() that gives an output 
</font>    <font color='#009900'>// value &lt;= -1 then it will stop immediately.  Usually you supply a number smaller than 
</font>    <font color='#009900'>// the actual global minimum.  So since the smallest output of the rosen function is 0 
</font>    <font color='#009900'>// we just put -1 here which effectively causes this last argument to be disregarded.
</font>
    <font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  <font color='#009900'>// Use BFGS search algorithm
</font>             <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>, <font color='#009900'>// Stop when the change in rosen() is less than 1e-7
</font>             rosen, rosen_derivative, starting_point, <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    <font color='#009900'>// Once the function ends the starting_point vector will contain the optimum point 
</font>    <font color='#009900'>// of (1,1).
</font>    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;


    <font color='#009900'>// Now let's try doing it again with a different starting point and the version
</font>    <font color='#009900'>// of find_min() that doesn't require you to supply a derivative function.  
</font>    <font color='#009900'>// This version will compute a numerical approximation of the derivative since 
</font>    <font color='#009900'>// we didn't supply one to it.
</font>    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#5555FF'>-</font><font color='#979000'>94</font>, <font color='#979000'>5.2</font><b>}</b>;
    <font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                           <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,
                                           rosen, starting_point, <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    <font color='#009900'>// Again the correct minimum point is found and stored in starting_point
</font>    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;


    <font color='#009900'>// Here we repeat the same thing as above but this time using the L-BFGS 
</font>    <font color='#009900'>// algorithm.  L-BFGS is very similar to the BFGS algorithm, however, BFGS 
</font>    <font color='#009900'>// uses O(N^2) memory where N is the size of the starting_point vector.  
</font>    <font color='#009900'>// The L-BFGS algorithm however uses only O(N) memory.  So if you have a 
</font>    <font color='#009900'>// function of a huge number of variables the L-BFGS algorithm is probably 
</font>    <font color='#009900'>// a better choice.
</font>    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>0.8</font>, <font color='#979000'>1.3</font><b>}</b>;
    <font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>,  <font color='#009900'>// The 10 here is basically a measure of how much memory L-BFGS will use.
</font>             <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>be_verbose</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  <font color='#009900'>// Adding be_verbose() causes a message to be 
</font>                                                                <font color='#009900'>// printed for each iteration of optimization.
</font>             rosen, rosen_derivative, starting_point, <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;

    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution: \n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;

    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#5555FF'>-</font><font color='#979000'>94</font>, <font color='#979000'>5.2</font><b>}</b>;
    <font color='#BB00BB'>find_min_using_approximate_derivatives</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>,
                                           <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,
                                           rosen, starting_point, <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution: \n</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;




    <font color='#009900'>// dlib also supports solving functions subject to bounds constraints on
</font>    <font color='#009900'>// the variables.  So for example, if you wanted to find the minimizer
</font>    <font color='#009900'>// of the rosen function where both input variables were in the range
</font>    <font color='#009900'>// 0.1 to 0.8 you would do it like this:
</font>    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>0.1</font>, <font color='#979000'>0.1</font><b>}</b>; <font color='#009900'>// Start with a valid point inside the constraint box.
</font>    <font color='#BB00BB'>find_min_box_constrained</font><font face='Lucida Console'>(</font><font color='#BB00BB'>lbfgs_search_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>10</font><font face='Lucida Console'>)</font>,  
                             <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>9</font><font face='Lucida Console'>)</font>,  
                             rosen, rosen_derivative, starting_point, <font color='#979000'>0.1</font>, <font color='#979000'>0.8</font><font face='Lucida Console'>)</font>;
    <font color='#009900'>// Here we put the same [0.1 0.8] range constraint on each variable, however, you
</font>    <font color='#009900'>// can put different bounds on each variable by passing in column vectors of
</font>    <font color='#009900'>// constraints for the last two arguments rather than scalars.  
</font>
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>constrained rosen solution: \n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;

    <font color='#009900'>// You can also use an approximate derivative like so:
</font>    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>0.1</font>, <font color='#979000'>0.1</font><b>}</b>; 
    <font color='#BB00BB'>find_min_box_constrained</font><font face='Lucida Console'>(</font><font color='#BB00BB'>bfgs_search_strategy</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  
                             <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>9</font><font face='Lucida Console'>)</font>,  
                             rosen, <font color='#BB00BB'>derivative</font><font face='Lucida Console'>(</font>rosen<font face='Lucida Console'>)</font>, starting_point, <font color='#979000'>0.1</font>, <font color='#979000'>0.8</font><font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>constrained rosen solution: \n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;




    <font color='#009900'>// In many cases, it is useful if we also provide second derivative information
</font>    <font color='#009900'>// to the optimizers.  Two examples of how we can do that are shown below.  
</font>    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>0.8</font>, <font color='#979000'>1.3</font><b>}</b>;
    <font color='#BB00BB'>find_min</font><font face='Lucida Console'>(</font><font color='#BB00BB'>newton_search_strategy</font><font face='Lucida Console'>(</font>rosen_hessian<font face='Lucida Console'>)</font>,
             <font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,
             rosen,
             rosen_derivative,
             starting_point,
             <font color='#5555FF'>-</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution: \n</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;

    <font color='#009900'>// We can also use find_min_trust_region(), which is also a method which uses
</font>    <font color='#009900'>// second derivatives.  For some kinds of non-convex function it may be more
</font>    <font color='#009900'>// reliable than using a newton_search_strategy with find_min().
</font>    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>0.8</font>, <font color='#979000'>1.3</font><b>}</b>;
    <font color='#BB00BB'>find_min_trust_region</font><font face='Lucida Console'>(</font><font color='#BB00BB'>objective_delta_stop_strategy</font><font face='Lucida Console'>(</font><font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>7</font><font face='Lucida Console'>)</font>,
                          <font color='#BB00BB'>rosen_model</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                          starting_point, 
                          <font color='#979000'>10</font> <font color='#009900'>// initial trust region radius
</font>    <font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>rosen solution: \n</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;





    <font color='#009900'>// Next, let's try the BOBYQA algorithm.  This is a technique specially
</font>    <font color='#009900'>// designed to minimize a function in the absence of derivative information.  
</font>    <font color='#009900'>// Generally speaking, it is the method of choice if derivatives are not available
</font>    <font color='#009900'>// and the function you are optimizing is smooth and has only one local optima.  As
</font>    <font color='#009900'>// an example, consider the be_like_target function defined below:
</font>    column_vector target <font color='#5555FF'>=</font> <b>{</b><font color='#979000'>3</font>, <font color='#979000'>5</font>, <font color='#979000'>1</font>, <font color='#979000'>7</font><b>}</b>;
    <font color='#0000FF'>auto</font> be_like_target <font color='#5555FF'>=</font> [<font color='#5555FF'>&amp;</font>]<font face='Lucida Console'>(</font><font color='#0000FF'>const</font> column_vector<font color='#5555FF'>&amp;</font> x<font face='Lucida Console'>)</font> <b>{</b>
        <font color='#0000FF'>return</font> <font color='#BB00BB'>mean</font><font face='Lucida Console'>(</font><font color='#BB00BB'>squared</font><font face='Lucida Console'>(</font>x<font color='#5555FF'>-</font>target<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>;
    starting_point <font color='#5555FF'>=</font> <b>{</b><font color='#5555FF'>-</font><font color='#979000'>4</font>,<font color='#979000'>5</font>,<font color='#979000'>99</font>,<font color='#979000'>3</font><b>}</b>;
    <font color='#BB00BB'>find_min_bobyqa</font><font face='Lucida Console'>(</font>be_like_target, 
                    starting_point, 
                    <font color='#979000'>9</font>,    <font color='#009900'>// number of interpolation points
</font>                    uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>1</font>, <font color='#5555FF'>-</font><font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,  <font color='#009900'>// lower bound constraint
</font>                    uniform_matrix<font color='#5555FF'>&lt;</font><font color='#0000FF'><u>double</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font><font color='#979000'>4</font>,<font color='#979000'>1</font>, <font color='#979000'>1e100</font><font face='Lucida Console'>)</font>,   <font color='#009900'>// upper bound constraint
</font>                    <font color='#979000'>10</font>,    <font color='#009900'>// initial trust region radius
</font>                    <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>6</font>,  <font color='#009900'>// stopping trust region radius
</font>                    <font color='#979000'>100</font>    <font color='#009900'>// max number of objective function evaluations
</font>    <font face='Lucida Console'>)</font>;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>be_like_target solution:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> starting_point <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;





    <font color='#009900'>// Finally, let's try the find_min_global() routine.  Like find_min_bobyqa(),
</font>    <font color='#009900'>// this technique is specially designed to minimize a function in the absence
</font>    <font color='#009900'>// of derivative information.  However, it is also designed to handle
</font>    <font color='#009900'>// functions with many local optima.  Where BOBYQA would get stuck at the
</font>    <font color='#009900'>// nearest local optima, find_min_global() won't.  find_min_global() uses a
</font>    <font color='#009900'>// global optimization method based on a combination of non-parametric global
</font>    <font color='#009900'>// function modeling and BOBYQA style quadratic trust region modeling to
</font>    <font color='#009900'>// efficiently find a global minimizer.  It usually does a good job with a
</font>    <font color='#009900'>// relatively small number of calls to the function being optimized.  
</font>    <font color='#009900'>// 
</font>    <font color='#009900'>// You also don't have to give it a starting point or set any parameters,
</font>    <font color='#009900'>// other than defining bounds constraints.  This makes it the method of
</font>    <font color='#009900'>// choice for derivative free optimization in the presence of multiple local
</font>    <font color='#009900'>// optima.  Its API also allows you to define functions that take a
</font>    <font color='#009900'>// column_vector as shown above or to explicitly use named doubles as
</font>    <font color='#009900'>// arguments, which we do here.
</font>    <font color='#0000FF'>auto</font> complex_holder_table <font color='#5555FF'>=</font> []<font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> x0, <font color='#0000FF'><u>double</u></font> x1<font face='Lucida Console'>)</font>
    <b>{</b>
        <font color='#009900'>// This function is a version of the well known Holder table test
</font>        <font color='#009900'>// function, which is a function containing a bunch of local optima.
</font>        <font color='#009900'>// Here we make it even more difficult by adding more local optima
</font>        <font color='#009900'>// and also a bunch of discontinuities. 
</font>
        <font color='#009900'>// add discontinuities
</font>        <font color='#0000FF'><u>double</u></font> sign <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
        <font color='#0000FF'>for</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> j <font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>4</font>; j <font color='#5555FF'>&lt;</font> <font color='#979000'>9</font>; j <font color='#5555FF'>+</font><font color='#5555FF'>=</font> <font color='#979000'>0.5</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>j <font color='#5555FF'>&lt;</font> x0 <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> x0 <font color='#5555FF'>&lt;</font> j<font color='#5555FF'>+</font><font color='#979000'>0.5</font><font face='Lucida Console'>)</font> 
                x0 <font color='#5555FF'>+</font><font color='#5555FF'>=</font> sign<font color='#5555FF'>*</font><font color='#979000'>0.25</font>;
            sign <font color='#5555FF'>*</font><font color='#5555FF'>=</font> <font color='#5555FF'>-</font><font color='#979000'>1</font>;
        <b>}</b>
        <font color='#009900'>// Holder table function tilted towards 10,10 and with additional
</font>        <font color='#009900'>// high frequency terms to add more local optima.
</font>        <font color='#0000FF'>return</font> <font color='#5555FF'>-</font><font face='Lucida Console'>(</font> std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>x0<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>cos</font><font face='Lucida Console'>(</font>x1<font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>exp</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>abs</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>std::<font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font>x0<font color='#5555FF'>*</font>x0<font color='#5555FF'>+</font>x1<font color='#5555FF'>*</font>x1<font face='Lucida Console'>)</font><font color='#5555FF'>/</font>pi<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font><font face='Lucida Console'>(</font>x0<font color='#5555FF'>+</font>x1<font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>10</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>x0<font color='#5555FF'>*</font><font color='#979000'>10</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>cos</font><font face='Lucida Console'>(</font>x1<font color='#5555FF'>*</font><font color='#979000'>10</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
    <b>}</b>;

    <font color='#009900'>// To optimize this difficult function all we need to do is call
</font>    <font color='#009900'>// find_min_global()
</font>    <font color='#0000FF'>auto</font> result <font color='#5555FF'>=</font> <font color='#BB00BB'>find_min_global</font><font face='Lucida Console'>(</font>complex_holder_table, 
                                  <b>{</b><font color='#5555FF'>-</font><font color='#979000'>10</font>,<font color='#5555FF'>-</font><font color='#979000'>10</font><b>}</b>, <font color='#009900'>// lower bounds
</font>                                  <b>{</b><font color='#979000'>10</font>,<font color='#979000'>10</font><b>}</b>, <font color='#009900'>// upper bounds
</font>                                  std::chrono::<font color='#BB00BB'>milliseconds</font><font face='Lucida Console'>(</font><font color='#979000'>500</font><font face='Lucida Console'>)</font> <font color='#009900'>// run this long
</font>                                  <font face='Lucida Console'>)</font>;

    cout.<font color='#BB00BB'>precision</font><font face='Lucida Console'>(</font><font color='#979000'>9</font><font face='Lucida Console'>)</font>;
    <font color='#009900'>// These cout statements will show that find_min_global() found the
</font>    <font color='#009900'>// globally optimal solution to 9 digits of precision:
</font>    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>complex holder table function solution y (should be -21.9210397): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> result.y <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>complex holder table function solution x:\n</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> result.x <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
<b>}</b>
<font color='#0000FF'>catch</font> <font face='Lucida Console'>(</font>std::exception<font color='#5555FF'>&amp;</font> e<font face='Lucida Console'>)</font>
<b>{</b>
    cout <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> e.<font color='#BB00BB'>what</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> endl;
<b>}</b>


</pre></body></html>